# -------------------------------------------------------------------
#       -*- coding: utf-8 -*-
#   @Project    :   spider
#   @File       :   44_北京新发地.py
#   @Author     :   WANGYU
#   @Time       :   2021-08-27 17:24:30
#   @Software   :   PyCharm
#   @Desc       :   多线程，线程池
# -------------------------------------------------------------------


# 如何提取单个页面到数据

import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
import csv

f = open('data.csv', mode='w', encoding='utf-8')
csvwriter = csv.writer(f)


def download_one_page(url):
    resp = requests.get(url)
    resp.encoding = 'utf-8'
    # print(resp.text)
    html = etree.HTML(resp.text)
    table = html.xpath('/html/body/div[2]/div[4]/div[1]/table')
    # print(table)
    trs = table.xpath('./tr')[1:]  # 切片，表头不要
    # trs = table.xpath('./tr【position()]>1') # 或者位置大于1
    # print(len(trs))
    # 拿到每个tr
    for tr in trs:
        txt = tr.xpath('./td/text()')
        # 对数据做简单对处理:去掉 // 、\
        txt = (item.replace('\\', '').replace('/', '') for item in txt)
        print(list(txt))
        # 把数据存放在文件中
        csvwriter.writerow(txt)
    print(url, '提取完毕！')


if __name__ == '__main__':
    # for i in range(1,14870):  #单线程，效率低
    #     download_one_page(f'http://www.xinfadi.com.cn/priceDetail{i}.html')

    with ThreadPoolExecutor(50) as t:
        for i in range(1,200):
            # 把下载任务提交给线程池
            t.submit(download_one_page,f'http://www.xinfadi.com.cn/priceDetail{i}.html')
    print('download all!')